{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Moving data from numpy arrays to pandas DataFrames\n",
    "In our last notebook we trained a model and compared our actual and predicted results\n",
    "\n",
    "What may not have been evident was when we did this we were working with two different objects: a **numpy array** and a **pandas DataFrame**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To explore further let's rerun the code from the previous notebook to create a trained model and get predicted values for our test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load our data from the csv file\n",
    "delays_df = pd.read_csv('Data/Lots_of_flight_data.csv') \n",
    "\n",
    "# Remove rows with null values since those will crash our linear regression model training\n",
    "delays_df.dropna(inplace=True)\n",
    "\n",
    "# Move our features into the X DataFrame\n",
    "X = delays_df.loc[:,['DISTANCE','CRS_ELAPSED_TIME']]\n",
    "\n",
    "# Move our labels into the y DataFrame\n",
    "y = delays_df.loc[:,['ARR_DELAY']] \n",
    "\n",
    "# Split our data into test and training DataFrames\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, \n",
    "                                                    y, \n",
    "                                                    test_size=0.3, \n",
    "                                                    random_state=42)\n",
    "regressor = LinearRegression()     # Create a scikit learn LinearRegression object\n",
    "regressor.fit(X_train, y_train)    # Use the fit method to train the model using your training data\n",
    "\n",
    "y_pred = regressor.predict(X_test)  # Generate predicted values for our test data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the last Notebook, you might have noticed the output displays differently when you display the contents of the predicted values in y_pred and the actual values in y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.47739078],\n",
       "       [5.89055919],\n",
       "       [4.33288464],\n",
       "       ...,\n",
       "       [5.84678979],\n",
       "       [6.05195889],\n",
       "       [5.66255414]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ARR_DELAY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>291483</th>\n",
       "      <td>-5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98997</th>\n",
       "      <td>-12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23454</th>\n",
       "      <td>-9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110802</th>\n",
       "      <td>-14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49449</th>\n",
       "      <td>-20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94944</th>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160885</th>\n",
       "      <td>-17.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47572</th>\n",
       "      <td>-20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164800</th>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62578</th>\n",
       "      <td>-9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196742</th>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91166</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171564</th>\n",
       "      <td>-9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60706</th>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240773</th>\n",
       "      <td>-6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32695</th>\n",
       "      <td>-13.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98399</th>\n",
       "      <td>-23.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167341</th>\n",
       "      <td>-11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126191</th>\n",
       "      <td>-4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188715</th>\n",
       "      <td>131.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258610</th>\n",
       "      <td>-5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215751</th>\n",
       "      <td>-20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41210</th>\n",
       "      <td>-15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68090</th>\n",
       "      <td>-19.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140794</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178840</th>\n",
       "      <td>-14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248071</th>\n",
       "      <td>21.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12770</th>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95948</th>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172913</th>\n",
       "      <td>-13.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200797</th>\n",
       "      <td>21.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36199</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70402</th>\n",
       "      <td>-37.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>285308</th>\n",
       "      <td>152.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201508</th>\n",
       "      <td>-2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154671</th>\n",
       "      <td>-5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238535</th>\n",
       "      <td>-5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133567</th>\n",
       "      <td>-9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3349</th>\n",
       "      <td>-8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257254</th>\n",
       "      <td>-28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106572</th>\n",
       "      <td>-19.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73023</th>\n",
       "      <td>-25.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214699</th>\n",
       "      <td>-12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274435</th>\n",
       "      <td>-7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67089</th>\n",
       "      <td>-10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269917</th>\n",
       "      <td>-4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164966</th>\n",
       "      <td>70.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>275120</th>\n",
       "      <td>-12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139292</th>\n",
       "      <td>-8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31106</th>\n",
       "      <td>-25.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>277799</th>\n",
       "      <td>17.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293749</th>\n",
       "      <td>-7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231114</th>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11645</th>\n",
       "      <td>-15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>252520</th>\n",
       "      <td>-12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209898</th>\n",
       "      <td>-20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22210</th>\n",
       "      <td>-9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165727</th>\n",
       "      <td>-6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260838</th>\n",
       "      <td>-33.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192546</th>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>88750 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        ARR_DELAY\n",
       "291483       -5.0\n",
       "98997       -12.0\n",
       "23454        -9.0\n",
       "110802      -14.0\n",
       "49449       -20.0\n",
       "...           ...\n",
       "209898      -20.0\n",
       "22210        -9.0\n",
       "165727       -6.0\n",
       "260838      -33.0\n",
       "192546        0.0\n",
       "\n",
       "[88750 rows x 1 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use **type()** to check the datatype of an object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* **y_pred** is a numpy array\n",
    "* **y_test** is a pandas DataFrame\n",
    "\n",
    "Another way you might discover this is if you try to use the **head** method on **y_pred**. \n",
    "\n",
    "This will return an error, because **head** is a method of the DataFrame class it is not a method of numpy arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'numpy.ndarray' object has no attribute 'head'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-9-05146ec42336>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0my_pred\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m: 'numpy.ndarray' object has no attribute 'head'"
     ]
    }
   ],
   "source": [
    "y_pred.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A one dimensional numpy array is similar to a pandas Series\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Pearson' 'Changi' 'Narita']\n",
      "Narita\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "airports_array = np.array(['Pearson','Changi','Narita'])\n",
    "print(airports_array)\n",
    "print(airports_array[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    Pearson\n",
      "1     Changi\n",
      "2     Narita\n",
      "dtype: object\n",
      "Narita\n"
     ]
    }
   ],
   "source": [
    "airports_series = pd.Series(['Pearson','Changi','Narita'])\n",
    "print(airports_series)\n",
    "print(airports_series[2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A two dimensional numpy array is similar to a pandas DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['YYZ' 'Pearson']\n",
      " ['SIN' 'Changi']\n",
      " ['NRT' 'Narita']]\n",
      "YYZ\n"
     ]
    }
   ],
   "source": [
    "airports_array = np.array([\n",
    "  ['YYZ','Pearson'],\n",
    "  ['SIN','Changi'],\n",
    "  ['NRT','Narita']])\n",
    "print(airports_array)\n",
    "print(airports_array[0,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     0        1\n",
      "0  YYZ  Pearson\n",
      "1  SIN   Changi\n",
      "2  NRT   Narita\n",
      "YYZ\n"
     ]
    }
   ],
   "source": [
    "airports_df = pd.DataFrame([['YYZ','Pearson'],['SIN','Changi'],['NRT','Narita']])\n",
    "print(airports_df)\n",
    "print(airports_df.iloc[0,0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you need the functionality of a DataFrame, you can move data from numpy objects to pandas objects and vice-versa.\n",
    "\n",
    "In the example below we use the DataFrame constructor to read the contents of the numpy array *y_pred* into a DataFrame called *predicted_df*\n",
    "\n",
    "Then we can use the functionality of the DataFrame object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.477391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.890559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.332885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.447476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.072394</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0\n",
       "0  3.477391\n",
       "1  5.890559\n",
       "2  4.332885\n",
       "3  3.447476\n",
       "4  5.072394"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicted_df = pd.DataFrame(y_pred)\n",
    "predicted_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
